package com.webRepo.spiderUtil;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.Map.Entry;
import java.util.concurrent.ArrayBlockingQueue;

public class RunTest {

    //收集爬取列表页匹配的内容
    private static volatile ArrayBlockingQueue<String> listPageDate = new ArrayBlockingQueue<String>(1000000);

    //收集爬取内容页匹配的内容
    private static volatile ArrayBlockingQueue<String> ContentPageDate = new ArrayBlockingQueue<String>(1000000);
    //需要爬的url队列
    private static volatile ArrayBlockingQueue<String> urlQueue = new ArrayBlockingQueue<String>(1000000);
    //所有url的前缀
    private static final String BASEURL = "http://www.xiamp4.com";
    //有规则的url模板
    private static String Spiderurl = BASEURL + "/GvodHtml/%s_%s.html";
    //列表页正则匹配规则
    static String listPageSplider = "<div class=\"info\"><h2><a href=\"/Html/\\w+.html\" title=\"([\u4e00-\u9fa5]|/|[\u4e00-\u9fa5])+\"";
    //内容页字符串匹配规则
    static String contenPageSplider = "var GvodUrls = ";

    @SuppressWarnings("unchecked")
    public static void main(String[] args) {
        try {
            //电影url
            for (int i = 2; i < 146; i++) {
                String url = String.format(Spiderurl, String.valueOf("1"),String.valueOf(i));
                urlQueue.add(url);
            }
            //电视剧url
            for (int i = 2; i < 487; i++) {
                String url = String.format(Spiderurl, String.valueOf("16"),String.valueOf(i));
                urlQueue.add(url);
            }
            //30个线程按照ListPageProcessor的实现依据listPageSplider规则对urlQueue进行并发爬取结果存入listPageDate中--列表页
            Spider.create(new ListPageProcessor(), listPageDate, listPageSplider).runReptile(urlQueue, 30);
            //将获取的列表页URL再次作为爬取的url进行内容页爬取
            Spider.create(new ContentPageProcessor(), ContentPageDate, contenPageSplider).runReptile(listPageDate, 30);
            
            //写文件
            File f = new File("d:/export");
            FileWriter fw = new FileWriter(f, true);
            BufferedWriter bf = new BufferedWriter(fw);
            while (!ContentPageDate.isEmpty()) {
                bf.write(ContentPageDate.poll());
                bf.newLine();
            }
            bf.flush();
            bf.close();
            System.out.println(Thread.currentThread().getName() + "存储完毕!");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
